# -*- coding:utf-8 -*-

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import os
import codecs

from storage import Storage


class Captrue:
    '''
    扒皮抓取内容逻辑
    '''

    __driver = ''
    url = '' # 需要抓取的链接
    headless = True #true 设置无头模式 false 不设置无头模式
    ip = '' #代理服务器地址
    port = '' #代理服务器端口
    __has = True #是否继续执行
    code = 0 #信息编号
    msg = '' #信息说明
    userName = '' #登陆账号
    passWord = '' #登陆密码
    userNameEle = {
        'type':1, #定位对象方式
        'tag':'', #定位对象标签
    }
    passWordEle = {
        'type':1, #定位对象方式
        'tag':'', #定位对象标签
    }
    loginBtnEle = {
        'type':1, #定位对象方式
        'tag':'', #定位对象标签
    }
    title = '' #抓取内容将要存储的文件名
    content = '' #抓取内容
    isInit = True #是否初始化了
    isWhile = True #是否循环执行
    coding = 'utf-8'
    num = 0 #循环的次数

    #初始化
    def __init__(self,url):
        if url:
            self.url = url
        else:
            self.__has = False
            self.code = 0
            self.msg = "抓取对象链接不能为空"


    def run(self):


        while self.isWhile:
            #self.num += 1
            self.buildDriver() #构建driver
            isBreak = self.__getUrl()
            if isBreak is False:
                break
            time.sleep(3)
            #页面渲染后的源代码
            self.content = self.__driver.page_source
            #替换资源中的静态文件
            #self.__replaceContent()
            #进行代码存储
            storage = Storage()
            extention = self.__extention()

            storage.save(self.content,self.title,extention)

            #检索源代码中是否存在下级链接存在进行存储 检索代码中是否存在js/css静态资源，若存在先进行本地已抓取的静态资源进行比对，比对不上的存储链接等待下一步抓取
            linkList = self.__hasLink()
            re = storage.save(linkList,'reserved','txt','reserved')
            if re:
                self.__driver.close() # 关闭当前页
                #删除执行后的那一行
                self.__delLine()

    #构建driver
    def buildDriver(self):
        profile=webdriver.FirefoxOptions()
        if self.headless:
            profile.add_argument('-headless') #设置无头模式

        #设置代理服务器
        profile.set_preference('network.proxy.type', 1)
        if self.ip != '':
            profile.set_preference('network.proxy.http',self.ip)#IP为你的代理服务器地址:如‘127.0.0.0’，字符串类型
        if self.port != '':
            profile.set_preference('network.proxy.http_port', self.port)  #PORT为代理服务器端口号:如，9999，整数类型
        self.__driver = webdriver.Firefox(options=profile)


    def __getUrl(self):
        if self.isInit:
             #第一执行
            self.isInit = False
        else:
            # 将从预抓取数据中获取url并开始循环抓取数据
            path = os.path.abspath(os.path.dirname(os.getcwd()))+'/reserved/reserved.txt'
            if os.path.exists(path):
                f = codecs.open(path,'r',self.coding)
                url = f.readline()
                #去除逗号
                if url == '' or url is None:
                    self.isWhile = False
                    return False
                print(url)
                self.url = url.strip()
                f.close()
        self.__driver.get(self.url)
        return True

    def __delLine(self):
        path = os.path.abspath(os.path.dirname(os.getcwd()))+'/reserved/reserved.txt'
        if os.path.exists(path):
            f = codecs.open(path,'r',self.coding)
            lines = f.readlines()
            f = codecs.open(path,'w',self.coding)
            for index,line in enumerate(lines):
               # print(lineStr)
               # print(self.url)
                if  index > 0:
                    f.write(line)
            f.close()

    #模拟登陆
    def login(self):

        if self.userName == '':
            self.code = 0
            self.msg = "登陆账号不能为空"
        if self.passWord == '':
            self.code = 0
            self.msg = "登陆密码不能为空"

        self.buildDriver() #构建driver
        #输入账号密码
        self.elementType(self.userNameEle['type'],self.userNameEle['tag']).send_keys(self.userName)
        self.elementType(self.passWordEle['type'],self.passWordEle['tag']).send_keys(self.passWord)

        #模拟点击登录
        self.elementType(self.loginBtnEle['type'],self.loginBtnEle['tag']).click()

    #获取定位对象的方式
    def elementType(self,type,tag):
        if type == 1:
            return self.__driver.find_element_by_id(tag)
        elif type == 2:
            return self.__driver.find_element_by_name(tag)
        elif type == 3:
            return  self.__driver.find_element_by_xpath(tag)
        elif type == 4:
            return self.__driver.find_element_by_link_text(tag)
        elif type == 5:
            return self.__driver.find_element_by_partial_link_text(tag)
        elif type == 6:
            return self.__driver.find_element_by_tag_name(tag)
        elif type == 7:
            return self.__driver.find_element_by_class_name(tag)
        elif type == 8:
            return self.__driver.find_element_by_css_selector(tag)
        elif type == 13:
            return  self.__driver.find_elements_by_xpath(tag)
        elif type == 14:
            return self.__driver.find_elements_by_link_text(tag)
        elif type == 15:
            return self.__driver.find_elements_by_partial_link_text(tag)
        elif type == 16:
            return self.__driver.find_elements_by_tag_name(tag)
        elif type == 17:
            return self.__driver.find_elements_by_class_name(tag)
        elif type == 18:
            return self.__driver.find_elements_by_css_selector(tag)

    def __extention(self):
        '''
        获取抓取链接 -- 若后缀是js,css将保存为此类文件
        若无后缀，将从内容中提起信息判断该文件信息
        '''

        url = self.url.replace("\\","/")

        url = url.lower()
        urlList = url.split('/')

        #获取最后的元素
        lastEle = urlList.pop()


        lastEle.rstrip('?')

        lastEleArr = lastEle.split('.')

        extention = lastEleArr.pop()
        if extention == 'css' or extention == 'js' or extention == 'html' or extention == 'html':
            self.title = lastEleArr[0] #文件标题
            return extention
        else:
            # 从内容中提取信息
            if self.content.find("html") > -1:
                self.title = lastEle
                return "html"

    def __hasLink(self):
        linkList = []
        eleList = self.elementType(16,'link')
        if eleList:
            for ele in eleList:
                string = ele.get_attribute("href")
                if string != '' and string is not None and string.find("http") > -1:
                    linkList.append(string)
        eleList = self.elementType(16,'script')
        if eleList:
            for ele in eleList:
                string = ele.get_attribute("src")
                if string != '' and string is not None and string.find("http") > -1:
                    linkList.append(string)
        #下级url
        eleList = self.elementType(16,'a')

        if eleList:
            for ele in eleList:
                string = ele.get_attribute("href")

                if string != '' and string is not None and string.find("http") > -1:
                    linkList.append(string)
        #去重
        linkList = list(set(linkList))
        return linkList

    def __replaceContent(self):
        '''
        替换原代码中的静态文件路径
        '''


    def __del__(self):
        self.__driver.quit()









a = Captrue("https://www.sfmj.cn/xwzx/636.html")
a.headless = True
a.run()



